#!/usr/bin/perl -w
use strict;
use XML::DOM;
use XML::LibXML;
use Data::Dumper;

# Grab the file from the command-prompt input:
my $xmlfile = shift;
die "Cannot find file \"$xmlfile\""
       unless -f $xmlfile;

my $parser = XML::LibXML->new();

my $doc = $parser->parse_file ($xmlfile);

my @nodes = $doc->getElementsByTagNameNS("*","CategoryScheme");


my $wordList = {};
my $lengths = {};

foreach my $node (@nodes) {
	my $numcats = 0;
	foreach my $category ($node->getElementsByTagNameNS("*","Category")) {
		my $categoryID = $category->getAttribute("id");
		my $title = $category->getElementsByTagNameNS('*', 'Label')->[0]->toString;
		my $desc = $category->getElementsByTagNameNS('*', 'Description')->[0]->toString;
		$desc =~ s/<(?:[^>'"]*|(['"]).*?\1)*>/ /gs;
		my @words = split(/\W+/, $desc);
		$lengths->{$categoryID} = length @words;
		foreach my $word (@words) {
			if (length $word > 2 and not bad($word)) {
				$wordList->{lc($word)}->{$categoryID} ++;
			}
		}
		@words = split(/\W+/, $title);
		foreach my $word (@words) {
			if (length $word > 2 and not bad($word)) {
				$word = standardise($word);
				$wordList->{lc($word)}->{$categoryID} += (100/@words);
			}
		}
	}
}

my $sql = "DROP TABLE IF EXISTS weights;

CREATE TABLE weights (
	word VARCHAR(255) NOT NULL,
	category VARCHAR(255) NOT NULL,
	name VARCHAR(255) NOT NULL,
	weight INT DEFAULT 0
);
";

foreach my $word (keys %$wordList) {
	foreach my $cat (keys %{$wordList->{$word}}) {
		my $weight = int (($wordList->{$word}->{$cat} * 1000) / (sqrt($lengths->{$cat} * 100) + (length (keys %{$wordList->{$word}}) * 10)));
		$sql .= "INSERT INTO weights (word, category, weight) VALUES ('$word','$cat','".$weight."');\n";
	}
}

#print Dumper($wordList);
print $sql;

exit(0);

#Checks a word against a list og know useless words
sub bad {
	my $word = shift;
	
	if (		$word =~ /^\d*$/
		or 	$word =~ /and/
	) { return 1 };

}

sub standardise {
	my $word = shift;
	
	if ($word !~ /ss$/) {
		$word =~ s/s$//;
	}

	return $word;
}
